﻿using Flurl.Http;
using Flurl.Http.Configuration;
using HtmlAgilityPack;
using ImageMagick;
using Microsoft.Data.Sqlite;
using System.Collections.Concurrent;
using System.Collections.Specialized;
using System.Data.SqlTypes;
using System.Net;
using Microsoft.Win32.SafeHandles;
using Url = Flurl.Url;
using DocumentFormat.OpenXml.Office.Word;

namespace Office.Automatic.Core.Jobs
{
    internal class JobReadFormT66Y :Job
    {
        internal class Post
        {
            public short Area { get; set; }
            public short Category { get; set; }
            public int Id { get; set; }
            public string Title { get; set; }
            public List<string> PhotoUrlList { get; }

            // ReSharper disable once InconsistentNaming
            public string RMLink { get; set; }

            public string Magnet { get; set; }

            public long Key => (long)Area << 48 | (long)Category << 32 | (long)Id;

            public Post(short area, short category, int id, string title, string rmLink)
            {
                Area = area;
                Category = category;
                Id=id; 
                Title = title;
                PhotoUrlList = new List<string>();
                RMLink = rmLink;
                Magnet = "";
            }
        }

        internal class PhotoLibrary: IDisposable
        {
            private FileStream _fileWrite;
            private FileStream _fileRead;
            private readonly BinaryReader _reader;
            private readonly BinaryWriter _writer;
            private readonly object _lock = new object();
            private bool _disposedValue;
            private const string FileHeader = "PIL_";
            private readonly string _filename;

            private readonly int _fileHeaderBytes =
                BitConverter.ToInt32(System.Text.Encoding.ASCII.GetBytes(FileHeader), 0);

            private int _fileVersion;
            private int _segmentCount;
            private int _photoCount;
            public int Count => _photoCount;
            private long[] _segmentOffsets = null!;
            private const int MaxSegmentSize = 0x10000;


            public PhotoLibrary(string filename)
            {
                _filename = filename;
                var newCreated = !File.Exists(filename);
                _fileWrite = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
                _fileRead = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                _reader = new BinaryReader(_fileRead, System.Text.Encoding.UTF8, true);
                _writer = new BinaryWriter(_fileWrite, System.Text.Encoding.UTF8, true);
                Initialize(newCreated);
            }

            private void CreateNew()
            {

                _fileVersion = 1;
                _segmentCount = 0;
                _photoCount = 0;
                _segmentOffsets = new long[MaxSegmentSize];

                _fileWrite.SetLength(524302);
                _fileWrite.Seek(0, SeekOrigin.Begin);
                _writer.Write(_fileHeaderBytes);
                _writer.Write(_fileVersion);
                _writer.Write((ushort)_segmentCount);
                _writer.Write(_photoCount);
            }

            private unsafe void OpenExist()
            {
                _fileWrite.Seek(0, SeekOrigin.Begin);

                var head = _reader.ReadInt32();
                if (head != _fileHeaderBytes)
                    throw new FormatException();

                _fileVersion = _reader.ReadInt32();
                if(_fileVersion!=1)
                    throw new FormatException();

                _segmentCount = _reader.ReadUInt16();
                _photoCount = _reader.ReadInt32();

                _segmentOffsets = new long[MaxSegmentSize];

                fixed (long* pSegOffsets = _segmentOffsets)
                {
                    var span = new Span<byte>(pSegOffsets, MaxSegmentSize * sizeof(long));
                    var bytesRead = _reader.Read(span);
                    if(bytesRead!= MaxSegmentSize * sizeof(long))
                        throw new FormatException();
                }
            }

            private void Initialize(bool createNew)
            {
                if (createNew) CreateNew();
                else OpenExist();
            }

            private long GetSegmentOffsetPtr(int segmentIndex)
            {
                return 14 + 8 * segmentIndex;
            }

            private long GetPhotoInfoPtr(int photoIndex)
            {
                var segmentIndex = photoIndex >> 8;

                if (segmentIndex >= _segmentCount)
                {
                    CreateSegment((ushort)segmentIndex);
                }

                var ptr = _segmentOffsets[segmentIndex] + 12 * (photoIndex & 0xff);

                return ptr;
            }

            

            private void UpdateSegmentCount(int segmentCount)
            {
                _fileWrite.Seek(8, SeekOrigin.Begin);
                _writer.Write((ushort)segmentCount);

                _segmentCount = segmentCount;
            }

            private void UpdatePhotoCount(int photoCount)
            {
                _fileWrite.Seek(10, SeekOrigin.Begin);
                _writer.Write(photoCount);

                _photoCount = photoCount;
            }

            private long CreateSegment(ushort segmentIndex)
            {
                var offset = _fileWrite.Length;
                _fileWrite.Seek(GetSegmentOffsetPtr(segmentIndex), SeekOrigin.Begin);
                _writer.Write(offset);

                // Area for photo data
                // { Offset:long(8), Length:uint(4) }
                _fileWrite.Seek(offset, SeekOrigin.Begin);
                _fileWrite.SetLength(offset + 3072/* 12 * 256 */);

                UpdateSegmentCount((segmentIndex + 1));

                _segmentOffsets[segmentIndex] = offset;

                return offset;
            }

            public int AppendPhoto(Stream stream)
            {
                lock (_lock)
                {
                    var photoIndex = _photoCount;

                    var photoInfoPtr = GetPhotoInfoPtr(photoIndex);

                    var fileLength = _fileWrite.Length;

                    _fileWrite.Seek(photoInfoPtr, SeekOrigin.Begin);
                    _writer.Write(fileLength);
                    _writer.Write((uint)stream.Length);

                    _fileWrite.Seek(fileLength, SeekOrigin.Begin);
                    stream.Seek(0,SeekOrigin.Begin);
                    stream.CopyTo(_fileWrite);

                    UpdatePhotoCount(_photoCount + 1);
                    return photoIndex;
                }
            }

            public Stream GetPhotoStream(int photoIndex)
            {
                if (photoIndex < 0 || photoIndex >= _photoCount)
                    throw new ArgumentOutOfRangeException(nameof(photoIndex));
                lock (_lock)
                {
                    var photoInfoPtr = GetPhotoInfoPtr(photoIndex);
                    _fileRead.Seek(photoInfoPtr, SeekOrigin.Begin);
                    var offset = _reader.ReadInt64();
                    var length= _reader.ReadUInt32();

                    return new PhotoStream(_filename, offset, (long)length);
                }
            }

            private class PhotoStream: Stream
            {
                private readonly Stream _baseStream;
                private readonly long _baseOffset;
                private readonly long _maxBasePosition;

                public PhotoStream(string filename, long offset, long length)
                {
                    _baseStream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                    _baseOffset = offset;
                    _maxBasePosition = _baseOffset + length;
                    Length = length;
                    _baseStream.Seek(_baseOffset, SeekOrigin.Begin);
                }
                public override void Flush()
                {
                    _baseStream.Flush();
                }

                public override int Read(byte[] buffer, int offset, int count)
                {
                    var position = Position;
                    
                    if (position + count > Length)
                        count = (int)(Length - position);

                    return _baseStream.Read(buffer, offset, count);
                }

                public override long Seek(long offset, SeekOrigin origin)
                {
                    switch (origin)
                    {
                        case SeekOrigin.Begin:
                        {
                            if (offset < 0) offset = 0;
                            var r = offset + _baseOffset;
                            if (r > _maxBasePosition) r = _maxBasePosition;
                            _baseStream.Seek(r, SeekOrigin.Begin);
                            return r - _baseOffset;
                        }
                        case SeekOrigin.Current:
                        {
                            var r = offset + _baseStream.Position;
                            if (r > _maxBasePosition) r = _maxBasePosition;
                            if (r < _baseOffset) r = _baseOffset;
                            _baseStream.Seek(r, SeekOrigin.Begin);
                            return r - _baseOffset;
                        }
                        case SeekOrigin.End:
                        {
                            if (offset < 0) offset = 0;
                            var r = _maxBasePosition - offset;
                            if (r < _baseOffset) r = _baseOffset;
                            _baseStream.Seek(r, SeekOrigin.Begin);
                            return r - _baseOffset;
                        }
                    }

                    return 0;
                }

                public override void SetLength(long value)
                {
                    throw new NotSupportedException();
                }

                public override void Write(byte[] buffer, int offset, int count)
                {
                    throw new NotSupportedException();
                }

                public override bool CanRead => true;
                public override bool CanSeek => true;
                public override bool CanWrite => false;
                public override long Length { get; }

                public override long Position
                {
                    get => _baseStream.Position - _baseOffset;
                    set => Seek(value, SeekOrigin.Begin);
                }
            }


            protected virtual void Dispose(bool disposing)
            {
                if (!_disposedValue)
                {
                    if (disposing)
                    {
                        _fileWrite.Dispose();
                    }

                    // TODO: 释放未托管的资源(未托管的对象)并重写终结器
                    _fileWrite = null!;
                    _disposedValue = true;
                }
            }

            // // TODO: 仅当“Dispose(bool disposing)”拥有用于释放未托管资源的代码时才替代终结器
            // ~PhotoLibrary()
            // {
            //     // 不要更改此代码。请将清理代码放入“Dispose(bool disposing)”方法中
            //     Dispose(disposing: false);
            // }

            public void Dispose()
            {
                // 不要更改此代码。请将清理代码放入“Dispose(bool disposing)”方法中
                Dispose(disposing: true);
                GC.SuppressFinalize(this);
            }
        }

        private class FlurlClientFactory : IFlurlClientFactory
        {
            private readonly ConcurrentDictionary<string, IFlurlClient> _clients = new();

            private readonly string _proxyUrl;
            public FlurlClientFactory(string proxyUrl)
            {
                _proxyUrl = proxyUrl;
            }

            public IFlurlClient Get(Url url)
            {
                if (url == null)
                {
                    throw new ArgumentNullException(nameof(url));
                }

                if (ShouldGoThroughProxy(url))
                {
                    string randomProxyUrl = ChooseRandomProxy();

                    return ProxiedClientFromCache(randomProxyUrl);
                }
                else
                {
                    return PerHostClientFromCache(url);
                }
            }

            private static bool ShouldGoThroughProxy(Url url)
            {
                return true;
            }

            private string ChooseRandomProxy()
            {
                return _proxyUrl;
            }

            private IFlurlClient PerHostClientFromCache(Url url)
            {
                return _clients.AddOrUpdate(
                    key: url.ToUri().Host,
                    addValueFactory: u => new FlurlClient(),
                    updateValueFactory: (u, client) => client.IsDisposed ? new FlurlClient() : client);
            }

            private IFlurlClient ProxiedClientFromCache(string proxyUrl)
            {
                return _clients.AddOrUpdate(
                    key: proxyUrl,
                    addValueFactory: u => CreateProxiedClient(proxyUrl),
                    updateValueFactory: (u, client) => client.IsDisposed ? CreateProxiedClient(proxyUrl) : client);
            }

            private IFlurlClient CreateProxiedClient(string proxyUrl)
            {
                HttpMessageHandler handler = new HttpClientHandler()
                {
                    Proxy = new WebProxy(proxyUrl),
                    UseProxy = true
                };

                var client = new HttpClient(handler);

                return new FlurlClient(client);
            }

            /// <summary>
            /// Disposes all cached IFlurlClient instances and clears the cache.
            /// </summary>
            public void Dispose()
            {
                foreach (var kv in _clients)
                {
                    if (!kv.Value.IsDisposed)
                        kv.Value.Dispose();
                }

                _clients.Clear();
            }
            
        }

        public override string Name => "Obtain data from t66y.com";
        public override bool IsTemporary => false;

        private const string UserAgent =
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.188";

        private const int MaxExistPostCountLimit = 20;

        private const int RequestRepeatDelay = 200;
        private const int PostRequestLimitPerDay = 2000;
        private volatile bool _cancelRequestPhotos = false;
        private volatile bool _procedureFinished = false;
        private readonly Queue<(long, long, int, string)> _pendingPhotoRequestQueue = new();
        private readonly object _pendingPhotoRequestQueueLock = new();
        private long _lastJobId = 0;
        private PhotoLibrary _photoLibrary;
        public override void Process()
        {
            _lastJobId = DateTime.Now.Ticks;
            _photoLibrary = new PhotoLibrary("photo-library.db");

            FlurlHttp.Configure(c =>
            {
                c.FlurlClientFactory = new FlurlClientFactory("http://127.0.0.1:19180");
            });

            var posts = new List<Post>();
            
            InitializeDb(out var conn);
           // UpdateDb(conn);

            Console.WriteLine("Photo request daemon started.");
            var daemon = StartPhotoRequestDaemon();
            var lastHandledPage = RequestPostList(1, 100, conn, out var totalPostAppended);



            _procedureFinished = true;

            daemon.Wait();

            ConsoleSuccess("Procedure succeed.\nStatistics: {0} pages with {1} posts processed.", null, lastHandledPage,
                totalPostAppended);

            ConsoleInfo("All procedure finished, press any key to exit.", null);
            Console.ReadKey();
        }

        private int ProcessPosts(List<Post> posts, SqliteConnection conn)
        {
            var postRemained = posts.Count;
            var appendPostCount = 0;
            foreach (var post in posts)
            {
                postRemained--;
                using var cmd = conn.CreateCommand();

                cmd.CommandText = "SELECT count(*) From Post WHERE Id=$id;";
                cmd.Parameters.AddWithValue("$id", post.Key);
                if (Convert.ToInt32(cmd.ExecuteScalar()) > 0)
                {
                    ConsoleSecondary(" #{0:x16} already exist, {1} posts remained.", "POST ", post.Key, postRemained);

                    continue;
                }

                try
                {
                    RequestPostData(post);

                    cmd.CommandText = "INSERT INTO Post(Id,Title,Magnet,Link) VALUES ($id,$title,$magnet,$link);";
                    cmd.Parameters.Clear();
                    cmd.Parameters.AddWithValue("$id", post.Key);
                    cmd.Parameters.AddWithValue("$title", post.Title);
                    cmd.Parameters.AddWithValue("$magnet", post.Magnet);
                    cmd.Parameters.AddWithValue("$link", post.RMLink);
                    cmd.ExecuteNonQuery();

                    QueuePhotoRequest(post.Key, post.PhotoUrlList, conn);

                    appendPostCount++;
                    ConsoleSuccess("#{0:x16} post data collected, will request {2} photos. ({1} posts remained)", "POST ",
                        post.Key, postRemained, post.PhotoUrlList.Count);
                }
                catch
                {
                    ConsoleCritical("#{0:x16} data collection failed, {1} posts remained..", "POST ", post.Key, postRemained);
                    continue;
                }
            }

            return appendPostCount;
        }

        private SqliteConnection OpenDatabase()
        {
            var connection = new SqliteConnection(new SqliteConnectionStringBuilder()
            {
                DataSource = "database.db",
                Pooling = true
            }.ConnectionString);
            connection.Open();

            return connection;
        }

        private void InitializeDb(out SqliteConnection connection)
        {
            Console.ForegroundColor = ConsoleColor.White;
            Console.Write("Initialize database...");

            connection = OpenDatabase();

            using var cmd = connection.CreateCommand();

            cmd.CommandText = "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='Post' ;";
            var c = Convert.ToInt32(cmd.ExecuteScalar());
            if (c == 0)
            {
                cmd.CommandText =
                    @"CREATE TABLE ""Post"" (
	""Id""	INTEGER NOT NULL UNIQUE,
	""Title""	TEXT NOT NULL,
	""Magnet""	TEXT NOT NULL,
	""Link""	TEXT NOT NULL,
	PRIMARY KEY(""Id""));";
                cmd.ExecuteNonQuery();

                cmd.CommandText = @"CREATE INDEX ""PostTitleIndex"" ON ""Post"" (""Title"" ASC);";
                cmd.ExecuteNonQuery();
            }

            cmd.CommandText = "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='Photo' ;";
            c = Convert.ToInt32(cmd.ExecuteScalar());
            if (c == 0)
            {
                cmd.CommandText =
                    @"CREATE TABLE ""Photo"" (""Url"" TEXT NOT NULL UNIQUE COLLATE NOCASE,""PhotoIndex"" INTEGER NOT NULL,PRIMARY KEY(""Url""));";
                cmd.ExecuteNonQuery();

                cmd.CommandText = @"CREATE INDEX ""PhotoIndexIndex"" ON ""Photo"" (""PhotoIndex"" ASC);";
                cmd.ExecuteNonQuery();
            }

            cmd.CommandText = "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='PostPhoto' ;";
            c = Convert.ToInt32(cmd.ExecuteScalar());
            if (c == 0)
            {
                cmd.CommandText =
                    @"CREATE TABLE ""PostPhoto"" (""PostId"" INTEGER NOT NULL,""PhotoIndex""	INTEGER NOT NULL);";
                cmd.ExecuteNonQuery();

                cmd.CommandText = @"CREATE INDEX ""PostPhotoIdIndex"" ON ""PostPhoto"" (""PostId"" ASC);";
                cmd.ExecuteNonQuery();

            }

            cmd.CommandText = "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='PhotoCache' ;";
            c = Convert.ToInt32(cmd.ExecuteScalar());
            if (c == 0)
            {
                cmd.CommandText =
                    @"CREATE TABLE ""PhotoCache"" (""JobId"" INTEGER NOT NULL UNIQUE,""PostId"" INTEGER NOT NULL,""PhotoIndex"" INTEGER NOT NULL,""PhotoUrl""	TEXT NOT NULL,""RequestAttempt"" INTEGER NOT NULL, PRIMARY KEY(""JobId""));";
                cmd.ExecuteNonQuery();
            }
            else
            {
                cmd.CommandText = @"SELECT JobId, PostId, PhotoIndex, PhotoUrl FROM PhotoCache;";
                using var reader = cmd.ExecuteReader();
                while (reader.Read())
                {
                    _pendingPhotoRequestQueue.Enqueue((reader.GetInt64(0), reader.GetInt64(1), reader.GetInt32(2),
                        reader.GetString(3)));
                }
            }

            Console.WriteLine("Ok");
            Console.WriteLine("{0} pending photo requests loaded.", _pendingPhotoRequestQueue.Count);
        }

        private void UpdateDb(SqliteConnection conn)
        {
            var photoList = new List<(int, string)>();

            using var cmd = conn.CreateCommand();

            cmd.CommandText =
                @"CREATE TABLE ""PhotoItem"" (""Url"" TEXT NOT NULL UNIQUE COLLATE NOCASE,""PhotoIndex"" INTEGER NOT NULL,PRIMARY KEY(""Url""));";
            cmd.ExecuteNonQuery();

            cmd.CommandText = @"CREATE INDEX ""PhotoIndexIndex"" ON ""PhotoItem"" (""PhotoIndex"" ASC);";
            cmd.ExecuteNonQuery();


            cmd.CommandText =
                @"CREATE TABLE ""PostPhotoIndex"" (""PostId"" INTEGER NOT NULL,""PhotoIndex""	INTEGER NOT NULL);";
            cmd.ExecuteNonQuery();

            cmd.CommandText = @"CREATE INDEX ""PostPhotoIdIndex"" ON ""PostPhotoIndex"" (""PostId"" ASC);";
            cmd.ExecuteNonQuery();


            cmd.CommandText = "SELECT rowid,Url FROM Photo;";
            using (var reader = cmd.ExecuteReader())
            {
                while (reader.Read())
                {
                    photoList.Add((reader.GetInt32(0), reader.GetString(1)));
                }

                reader.Close();
            }

            Console.WriteLine("{0} photos found.", photoList.Count);
            var photoUrlIndexMap = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);

            using (var transaction = conn.BeginTransaction())
            {
                foreach (var (rowId, url) in photoList)
                {
                    int photoIndex;
                    using (var blob = new SqliteBlob(conn, "Photo", "Data", rowId, true))
                    {
                        photoIndex = _photoLibrary.AppendPhoto(blob);
                    }

                    cmd.CommandText = "INSERT INTO PhotoItem(Url, PhotoIndex) VALUES ($url, $index);";
                    cmd.Transaction = transaction;
                    cmd.Parameters.Clear();
                    cmd.Parameters.AddWithValue("$url", url);
                    cmd.Parameters.AddWithValue("$index", photoIndex);
                    cmd.ExecuteNonQuery();

                    Console.WriteLine("Photo #{0} from {1} processed.", photoIndex, url);
                    photoUrlIndexMap.Add(url, photoIndex);
                }

                transaction.Commit();
            }

            photoList = null;

            var postToPhotoList=new List<(long, string)>();

            cmd.CommandText = "SELECT * FROM PostPhoto;";
            cmd.Transaction = null;
            using (var reader = cmd.ExecuteReader())
            {
                while (reader.Read())
                    postToPhotoList.Add((reader.GetInt64(0), reader.GetString(1)));
            }

            using (var transaction = conn.BeginTransaction())
            {
                foreach (var (postId, url) in postToPhotoList)
                {
                    var photoIndex = photoUrlIndexMap[url];
                    cmd.CommandText = "INSERT INTO PostPhotoIndex(PostId,PhotoIndex) VALUES ($postId,$photoIndex);";
                    cmd.Transaction = transaction;
                    cmd.Parameters.Clear();
                    cmd.Parameters.AddWithValue("$postId", postId);
                    cmd.Parameters.AddWithValue("$photoIndex", photoIndex);
                }
            }

            cmd.Transaction = null;
            cmd.CommandText = @"DROP TABLE PostPhoto; 
DROP TABLE Photo;
ALTER TABLE PostPhotoIndex RENAME TO ""PostPhoto"";
ALTER TABLE PhotoItem RENAME TO ""Photo""";
            cmd.ExecuteNonQuery();

            Console.WriteLine("Database update finished.");

        }

        private int RequestPostList(int startPageNum, int maxPageNum,SqliteConnection conn, out int totalPostAppended)
        { 
            totalPostAppended = 0;
            if (maxPageNum < 1) throw new ArgumentException("Page Number should be large than 0.", nameof(maxPageNum));

            for (var page = startPageNum; page <= maxPageNum; page++)
            {
                var posts = new List<Post>();

                var url = $"https://www.t66y.com/thread0806.php?fid=25&search=&page={page}";

                string html;
                try
                {
                    var resp = url.WithHeader("User-Agent", UserAgent).GetAsync().Result;
                    html = resp.GetStringAsync().Result;
                }
                catch
                {
                    ConsoleWarn("T66y service is current not available.","List ");
                    return page;
                }

                var doc = new HtmlDocument();
                doc.LoadHtml(html);

                var nodes = doc.DocumentNode.SelectNodes("//table[@id=\"ajaxtable\"]/tbody[@id=\"tbody\"]/tr");

                var count = 0;
                foreach (var node in nodes)
                {
                    var nodeLink = node.SelectSingleNode("td[2]/h3/a");
                    if (nodeLink == null) continue;

                    var segments = nodeLink.Attributes["href"]?.Value.Split('/');
                    if (segments == null || segments.Length < 3) continue;

                    var title = nodeLink.InnerText;

                    posts.Add(new Post(Convert.ToInt16(segments[1]), Convert.ToInt16(segments[2]),
                        Convert.ToInt32(segments[3].Substring(0, segments[3].Length-5)), title, string.Empty));
                    count++;
                }
                
                ConsolePost("{0} posts collected from page {1}.", "List ", count, page);

                var processedPost = ProcessPosts(posts, conn);
                totalPostAppended += processedPost;

                // if (posts.Count - processedPost >= MaxExistPostCountLimit)
                // { 
                //     ConsoleSuccess("Post list is up to date, {0} posts update.", null, totalProcessedPostCount);
                //     return page;
                // }


                if (totalPostAppended >= PostRequestLimitPerDay)
                {
                    ConsoleWarn("Max post request limit reached, {0} posts update.", null, totalPostAppended);
                    return page;
                }

                Thread.Sleep(RequestRepeatDelay);
            }

            return maxPageNum;
        }

        private void RequestPostData(Post post)
        {
            
            var url = $"https://www.t66y.com/htm_data/{post.Area}/{post.Category}/{post.Id}.html";
            var html = url.WithHeader("User-Agent", UserAgent).GetStringAsync().Result;
            var doc = new HtmlDocument();
            doc.LoadHtml(html);

            var contentNode = doc.DocumentNode.SelectSingleNode("//div[@class=\"tpc_content do_not_catch\"]");

            var photoNodes = contentNode.SelectNodes(".//img");
            if(photoNodes != null)
                foreach (var photo in photoNodes)
                {
                    var photoUrl = photo.Attributes["ess-data"].Value;
                    if (photoUrl == null)
                        continue;

                    post.PhotoUrlList.Add(photoUrl);
                }

            var linkNodes = contentNode.SelectNodes(".//a");
            foreach (var link in linkNodes)
            {
                var linkUrl = link.Attributes["href"]?.Value;
                if (linkUrl != null && linkUrl.Contains("rmdown"))
                {
                    post.RMLink = linkUrl;
                    break;
                }
            }

            Thread.Sleep(RequestRepeatDelay);
        }

        private int RequestPhotoData(long postId, int index, string url, SqliteConnection conn, out int photoIndex)
        {
            var response = url.AllowHttpStatus("400-404,6xx").WithHeader("User-Agent", UserAgent).WithTimeout(40).GetAsync();

            photoIndex = -1;

            try
            {
                if (response.Result.StatusCode != 200)
                    return 1;
            }
            catch(Exception ex)
            {
                return 1;
            }

            var stream = response.ReceiveStream().Result;
            if (stream.Length == 0)
                return 1;

            using var image = new MagickImage(stream);
            var radio = (image.Width + 0.0d) / (image.Height + 0.0d);
            if (radio > 5.0d)
            {
                // Maybe advertisement.
                return -1;
            }

            
            photoIndex = _photoLibrary.AppendPhoto(stream);

            bool exist;
            using (var cmd = conn.CreateCommand())
            {
                cmd.CommandText =
                    "SELECT count(*) FROM Photo WHERE Url=$url;";
                cmd.Parameters.AddWithValue("$url", url);

                exist = Convert.ToInt32(cmd.ExecuteScalar()) > 0;
                
            }

            if(!exist)
            {
                using var cmd = conn.CreateCommand();
                cmd.CommandText =
                    "INSERT INTO Photo(Url,PhotoIndex) VALUES($url,$index);";
                cmd.Parameters.AddWithValue("$url", url);
                cmd.Parameters.AddWithValue("$index", photoIndex);
                cmd.ExecuteNonQuery();
            }
            
            
            var filename = $"C:\\Users\\loyse\\Desktop\\Workspace\\Tmp\\{postId:x16}-{index:D2}";
            switch (image.Format)
            {
                case MagickFormat.Png:
                    image.Write(filename + ".png");
                    break;
                case MagickFormat.Gif:
                    image.Write(filename + ".gif");
                    break;
                case MagickFormat.Jpg:
                    image.Write(filename + ".jpg");
                    break;
                case MagickFormat.Jpeg:
                    image.Write(filename + ".jpg");
                    break;
            }

            ConsoleInfo("New photo added from {0}", "Lib  ", url);
            return 0;
        }

        private void QueuePhotoRequest(long postId, IEnumerable<string> urls, SqliteConnection conn)
        {
            using var cmd = conn.CreateCommand();
        
            var index = 1;
            foreach (var url in urls)
            {
                cmd.CommandText =
                    "INSERT INTO PhotoCache(JobId, PostId, PhotoIndex, PhotoUrl,RequestAttempt) VALUES ($jobid,$postid,$index,$url,0);";
                cmd.Parameters.Clear();
                cmd.Parameters.AddWithValue("$jobid", _lastJobId);
                cmd.Parameters.AddWithValue("$postid", postId);
                cmd.Parameters.AddWithValue("$index", index);
                cmd.Parameters.AddWithValue("$url", url);
                cmd.ExecuteNonQuery();

                lock (_pendingPhotoRequestQueueLock)
                    _pendingPhotoRequestQueue.Enqueue((_lastJobId, postId, index, url));

                index++;
                _lastJobId++;
            }
            
        }


        private readonly object _daemonDbLocker = new();
        private volatile int _daemonRequestCount = 0;
        private const int MaxDaemonRequest = 19;
        private async Task StartPhotoRequestDaemon()
        {
            _cancelRequestPhotos = false;

            await Task.Run(() =>
            {
                var conn = new SqliteConnection(new SqliteConnectionStringBuilder()
                {
                    DataSource = "database.db",
                    Pooling = true
                }.ConnectionString);
                conn.Open();

                while (_cancelRequestPhotos == false)
                {
                    (long, long, int, string) item;
                    
                    Monitor.Enter(_pendingPhotoRequestQueueLock);
                    var reqCount = _pendingPhotoRequestQueue.Count;

                    if (reqCount > 0 && _daemonRequestCount < MaxDaemonRequest)
                    {
                        item = _pendingPhotoRequestQueue.Dequeue();
                        Monitor.Exit(_pendingPhotoRequestQueueLock);

                        Task.Run(() =>
                        {
                            Interlocked.Increment(ref _daemonRequestCount);
                            RequestPhoto(item.Item1, item.Item2, item.Item3, item.Item4);
                            Interlocked.Decrement(ref _daemonRequestCount);
                        });
                    }
                    else
                    {
                        Monitor.Exit(_pendingPhotoRequestQueueLock);

                        if (reqCount == 0 && _procedureFinished){
                            while (_daemonRequestCount > 0)
                            {
                                Thread.Sleep(50);
                            }

                            break;
                        }
                        Thread.Sleep(50);
                        continue;
                    }

                    Thread.Sleep(100);
                }

            });
        }

        private readonly object _consoleOutputLocker = new();


        private void ConsoleSuccess(string message, string? predicate, params object[] args)
        {
            lock (_consoleOutputLocker)
            {
                Console.ForegroundColor = ConsoleColor.Green;
                if (predicate != null)
                    Console.Write("[{0}]", predicate);
                Console.WriteLine(message, args);
            }
        }

        private void ConsolePost(string message, string? predicate, params object[] args)
        {
            lock (_consoleOutputLocker)
            {
                Console.ForegroundColor = ConsoleColor.Blue;
                if (predicate != null)
                    Console.Write("[{0}]", predicate);
                Console.WriteLine(message, args);
            }
        }

        private void ConsoleInfo(string message, string? predicate, params object[] args)
        {
            lock (_consoleOutputLocker)
            {
                Console.ForegroundColor = ConsoleColor.Gray;
                if (predicate != null)
                    Console.Write("[{0}]", predicate);
                Console.WriteLine(message, args);
            }
        }

        private void ConsoleImportant(string message, string? predicate, params object[] args)
        {
            lock (_consoleOutputLocker)
            {
                Console.ForegroundColor = ConsoleColor.White;
                if (predicate != null)
                    Console.Write("[{0}]", predicate);
                Console.WriteLine(message, args);
            }
        }

        private void ConsoleWarn(string message, string? predicate, params object[] args)
        {
            lock (_consoleOutputLocker)
            {
                Console.ForegroundColor = ConsoleColor.Yellow;
                if (predicate != null)
                    Console.Write("[{0}]", predicate);
                Console.WriteLine(message, args);
            }
        }

        private void ConsoleFatal(string message, string? predicate, params object[] args)
        {
            lock (_consoleOutputLocker)
            {
                Console.ForegroundColor = ConsoleColor.Red;
                if (predicate != null)
                    Console.Write("[{0}]", predicate);
                Console.WriteLine(message, args);
            }
        }

        private void ConsoleCritical(string message, string? predicate, params object[] args)
        {
            lock (_consoleOutputLocker)
            {
                Console.ForegroundColor = ConsoleColor.DarkRed;
                if (predicate != null)
                    Console.Write("[{0}]", predicate);
                Console.WriteLine(message, args);
            }
        }

        private void ConsoleSecondary(string message, string? predicate, params object[] args)
        {
            lock (_consoleOutputLocker)
            {
                Console.ForegroundColor = ConsoleColor.DarkGray;
                if (predicate != null)
                    Console.Write("[{0}]", predicate);
                Console.WriteLine(message, args);
            }
        }

        private void RequestPhoto(long jobId, long postId, int photoIndex, string url)
        {
            bool photoExist;
            var photoLibraryIndex = -1;

            var conn = OpenDatabase();

            using (var cmd = conn.CreateCommand())
            {
                cmd.CommandText = "SELECT PhotoIndex FROM Photo Where Url=$url;";
                cmd.Parameters.Clear();
                cmd.Parameters.AddWithValue("$url", url);
                var result = cmd.ExecuteScalar();
                photoExist = result != null;
                if (photoExist) photoLibraryIndex = Convert.ToInt32(result);
            }

            // Check whether this photo already exist.
            if (!photoExist)
            {
                var requestResult = RequestPhotoData(postId, photoIndex, url, conn, out photoLibraryIndex);
                if (requestResult == 1)
                {
                    // photo inaccessible
                    ConsoleCritical("Photo inaccessible for #{0:x16} from {1}, {2} photo requests remained.", "Photo",
                        postId, url, _pendingPhotoRequestQueue.Count);

                    
                    using var cmd = conn.CreateCommand();
                    cmd.CommandText = "SELECT RequestAttempt FROM PhotoCache WHERE JobId=$id;";
                    cmd.Parameters.AddWithValue("$id", jobId);
                    var attempt = Convert.ToInt32(cmd.ExecuteScalar());

                    if (attempt > 10)
                    {
                        cmd.CommandText = "DELETE FROM PhotoCache WHERE JobId=$id;";
                        cmd.Parameters.Clear();
                        cmd.Parameters.AddWithValue("$id", jobId);
                        cmd.ExecuteNonQuery();

                        ConsoleWarn("Cancel photo request for url {0}, to many fails.", "Photo", url);
                    }
                    else
                    {
                        cmd.CommandText = "UPDATE PhotoCache SET RequestAttempt=$attempt WHERE JobId=$id;";
                        cmd.Parameters.Clear();
                        cmd.Parameters.AddWithValue("$id", jobId);
                        cmd.Parameters.AddWithValue("$attempt", attempt + 1);
                        cmd.ExecuteNonQuery();
                    }

                    return;

                }else if (requestResult == -1)
                {
                    // photo not match
                    using var cmd = conn.CreateCommand();
                    cmd.CommandText = "DELETE FROM PhotoCache WHERE JobId=$id;";
                    cmd.Parameters.Clear();
                    cmd.Parameters.AddWithValue("$id", jobId);
                    cmd.ExecuteNonQuery();

                    return;
                }
            }

            
            bool exist;
            using (var cmd = conn.CreateCommand())
            {
                cmd.CommandText = "SELECT count(*) FROM PostPhoto WHERE PostId=$id AND PhotoIndex=$index;";
                cmd.Parameters.Clear();
                cmd.Parameters.AddWithValue("$id", postId);
                cmd.Parameters.AddWithValue("$index", photoLibraryIndex);
                exist = Convert.ToInt32(cmd.ExecuteScalar()) > 0;
            }

            if (!exist)
            {
                using var cmd = conn.CreateCommand();
                cmd.CommandText = "INSERT INTO PostPhoto(PostId,PhotoIndex) VALUES($id,$index);";
                cmd.Parameters.Clear();
                cmd.Parameters.AddWithValue("$id", postId);
                cmd.Parameters.AddWithValue("$index", photoLibraryIndex);
                cmd.ExecuteNonQuery();
            }

            using (var cmd = conn.CreateCommand())
            {

                cmd.CommandText = "DELETE FROM PhotoCache WHERE JobId=$id;";
                cmd.Parameters.Clear();
                cmd.Parameters.AddWithValue("$id", jobId);
                cmd.ExecuteNonQuery();
            }
            

            ConsoleImportant("New photo({2} remained) collected for #{0:x16} at {3}, {1} requests alive.", "Photo",
                postId,
                _daemonRequestCount, _pendingPhotoRequestQueue.Count, url);
        }
        

    }
}
